import requests
import time
import json
from bs4 import BeautifulSoup

def get_message(Url, header, pages):
    messsage_collect = []
    save_txt_name=''
    for n in range(pages):
        html = requests.get(url=Url, headers=header)
        soup = BeautifulSoup(html.text, 'lxml')
        house_list = soup.find('ul', id='houselist-mod-new')
        save_txt_name = soup.select_one('.selected-item').text

        for i in house_list.select('.list-item'):
            dic = {}
            dic['标题'] = i.select_one('.house-title').text  # 房子标题

            dic['基本信息'] = []
            for j in i.select_one('.details-item').find_all('span'):  # 房子基本信息
                dic['基本信息'].append(j.text)

            dic['位置'] = i.select_one('.comm-address').text  # 房子位置

            dic['评价标签'] = []
            for k in i.select_one('.tags-bottom').find_all('span'):  # 房子评价标签
                dic['评价标签'].append(k.text)

            dic['总价格'] = i.select_one('.price-det').text  # 总房价
            dic['单位面积价格'] = i.select_one('.unit-price').text  # 单位面积价格
            messsage_collect.append(dic)
        try:
            Url = soup.select_one('.aNxt')['href']
        except:
            break
        print(save_txt_name+'成功爬取第' + str(n + 1) + '页')
        time.sleep(0.2)

    with open(save_txt_name+'.json', 'w', encoding='utf-8') as f:
        json.dump(messsage_collect, f, ensure_ascii=0, indent=4)

    print(save_txt_name+'爬虫成功')

def main():
    send_header = {'user-agent': "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko)",
                   'cookie': 'wmda_uuid=74ba33008e705d1e05f429118ce66261; wmda_new_uuid=1; wmda_visited_projects=%3B6289197098934; als=0; isp=true; Hm_lvt_c5899c8768ebee272710c9c5f365a6d8=1555132061,1555132095,1555132292,1555392964; ctid=34; sessid=D8D3E368-663E-0FB7-D203-B3DB3CB79A6F; lps=http%3A%2F%2Fdg.anjuke.com%2Fsale%2Fchangpingb%2F%7C; _gat=1; __xsptplusUT_8=1; __xsptplus8=8.5.1555392942.1555393910.3%233%7Cwww.uc123.com%7C%7C%7C%7C%23%23SmmkLHoGcSlgkQmrxOzvGTuWHBBcFUTM%23; wmda_session_id_6289197098934=1555390127990-2e3fd7c7-2cf9-714b; _ga=GA1.2.2075413695.1554607749; _gid=GA1.2.739811232.1555304108; 58tj_uuid=04246483-cfd8-40b4-81de-f70a9da6e9de; new_session=0; init_refer=; new_uv=7; aQQ_ajkguid=13E85FCD-D847-D8C2-322C-7AB247D79FC4; twe=2'
                   }
    urls=['https://dg.anjuke.com/sale/',
          'https://dg.anjuke.com/sale/qitadg/',
          'https://dg.anjuke.com/sale/nanchenga/',
          'https://dg.anjuke.com/sale/zhangmutou/',
          'https://dg.anjuke.com/sale/humen/',
          'https://dg.anjuke.com/sale/tangxia/',
          'https://dg.anjuke.com/sale/dongchengd/',
          'https://dg.anjuke.com/sale/changpingb/',
          'https://dg.anjuke.com/sale/fengganga/',
          'https://dg.anjuke.com/sale/changanc/',
          'https://dg.anjuke.com/sale/dalang/',
          'https://dg.anjuke.com/sale/wanjianga/',
          'https://dg.anjuke.com/sale/liaobua/',
          'https://dg.anjuke.com/sale/huangjianga/',
          'https://dg.anjuke.com/sale/songshanhua/',
          'https://dg.anjuke.com/sale/guanchengb/',
          'https://dg.anjuke.com/sale/shatianzhen/',
          'https://dg.anjuke.com/sale/qingxizhen/',
          'https://dg.anjuke.com/sale/dalinshanzhen/',
          'https://dg.anjuke.com/sale/zhongtangzhen/',
          'https://dg.anjuke.com/sale/qiaotouzhen/',
          'https://dg.anjuke.com/sale/shilongzhen/',
          'https://dg.anjuke.com/sale/shijiezhen/',
          'https://dg.anjuke.com/sale/henglizhen/',
          'https://dg.anjuke.com/sale/daojiaozhen/',
          'https://dg.anjuke.com/sale/chashanzhen/',
          'https://dg.anjuke.com/sale/mayongzhen/',
          'https://dg.anjuke.com/sale/shipaizhen/',
          'https://dg.anjuke.com/sale/dongkengzhen/',
          'https://dg.anjuke.com/sale/gaobuzhen/',
          'https://dg.anjuke.com/sale/qishizhen/',
          'https://dg.anjuke.com/sale/xiegangzhen/',
          'https://dg.anjuke.com/sale/hongmeizhen/',
          'https://dg.anjuke.com/sale/dgwangniudunzhen/']
    for u in urls:
        get_message(Url=u, header=send_header, pages=100)
if __name__ == '__main__':
    main()